# Import pickle files of train, validation, and test sets:
import pickle
import numpy as np
import pandas as pd
training_file = 'traffic-signs-data/train.p'
validation_file = 'traffic-signs-data/valid.p'
testing_file = 'traffic-signs-data/test.p'
with open(training_file, mode='rb') as f:
train = pickle.load(f)
with open(validation_file, mode='rb') as f:
valid = pickle.load(f)
with open(testing_file, mode='rb') as f:
test = pickle.load(f)
X_train, y_train = train['features'], train['labels']
X_valid, y_valid = valid['features'], valid['labels']
X_test, y_test = test['features'], test['labels']
# Load sign names:
sign_names = pd.read_csv('signnames.csv')[['ClassId', 'SignName']].values
# Save original sizes and bounding-box coordinates (this will help during preprocessing):
train_sizes= train['sizes']
train_coord = train['coords']
valid_sizes= valid['sizes']
valid_coord = valid['coords']
test_sizes= test['sizes']
test_coord = test['coords']
n_train = X_train.shape[0]
n_validation = X_valid.shape[0]
n_test = X_test.shape[0]
image_shape = X_train.shape[1:]
n_classes = len(np.unique(y_train))
print("Number of training examples =", n_train)
print("Number of validation examples =", n_validation)
print("Number of testing examples =", n_test)
print("Image data shape =", image_shape)
print("Number of classes =", n_classes)
I like looking at the numbers rather than a graph. I find it easier to ingest the information, so that's what I'll do.
a, b = np.unique(y_train, return_counts=True)
print('Training Data:')
for i in range(len(a)):
print('{:.20} occurs {} times or {:.0f}% of total'.format(sign_names[a[i], 1], b[i], (b[i]/np.sum(b))*100))
print('')
a, b = np.unique(y_valid, return_counts=True)
print('Validation Data:')
for i in range(len(a)):
print('{:.20} occurs {} times or {:.0f}% of total'.format(sign_names[a[i], 1], b[i], (b[i]/np.sum(b))*100))
print('')
a, b = np.unique(y_test, return_counts=True)
print('Test Data:')
for i in range(len(a)):
print('{:.30} occurs {} times or {:.0f}% of total'.format(sign_names[a[i], 1], b[i], (b[i]/np.sum(b))*100))
Nothing too surprising. The distributions look similar. In all three sets, some signs occur more than others. Some of the notable ones are speed-limit, priority road, and yield signs.
I'll start by printing 80 random signs from each set.
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
ii = 1
idx = np.arange(n_train)
np.random.shuffle(idx)
plt.figure(figsize=(45, 45))
for i in range(10*8):
plt.subplot(2*4,5*2,(i+1))
plt.title('{:.20}'.format(sign_names[y_train[idx[i*ii]], 1]), fontsize=22)
plt.imshow(X_train[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
ii = 1
idx = np.arange(n_validation)
np.random.shuffle(idx)
plt.figure(figsize=(45, 45))
for i in range(10*8):
plt.subplot(2*4,5*2,(i+1))
plt.title('{:.33}'.format(sign_names[y_valid[idx[i*ii]], 1]), fontsize=22)
plt.imshow(X_valid[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
ii = 1
idx = np.arange(n_test)
np.random.shuffle(idx)
plt.figure(figsize=(45, 45))
for i in range(10*8):
plt.subplot(2*4,5*2,(i+1))
plt.title('{:.33}'.format(sign_names[y_test[idx[i*ii]], 1]), fontsize=22)
plt.imshow(X_test[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
plt.figure(figsize=(15,15))
for i in range(9):
plt.subplot(3,3,(i+1))
plt.imshow(X_train[4200+i])
plt.title(sign_names[y_train[4200+i], 1])
As it happens, the below sign was the most misclassified sign for a network that I ran for 200 epochs and achieved a validation accuracy over 95%. Too bad it repeats many times within the validaiton set, so the one mistake counts as 10+ mistakes :(
plt.figure(figsize=(15,15))
for i in range(9):
plt.subplot(3,3,(i+1))
plt.imshow(X_valid[540+i])
plt.title(sign_names[y_valid[540+i], 1])
The first five functions augment the training data and are not applied during test time on the validation or test sets. They are applied randomly to each image in each batch so the network never sees the exact same image twice, even if there are repeats :)
random_blur: Applies a Gaussian Noise kernel 50% of time
random_brighten: After noticing that signs in all sets had varying degrees of brightness, which didn't seem to be related to natural light conditions, I used this to randomly brighten or darken an image in a way to resemble images in the validation and test sets. I found out about OpenCV's lookup table from this tutorial.
random_red_adjust: This is poorly named because it adjusts all three color channels. It started out only affecting the red channel, but seemed to help the network generalize, so evolved.
random_rotate: Self-explanatory. I don't flip images and only rotate them by a max of 40 degrees because certain signs, like Left Turn Ahead and speed-limit signs, would confuse the network if flipped.
random_crop: Self-explanatory.
The only preprocessing normalization is to scale the data between 0 and 1.
Since bounding boxes are included, every image is cropped to include only the region within the box and resized to 32x32x3 for uniformity.
Images are kept in color because networks trained on RGB-images performed better for me than those on grayscale. Intuitively, this makes sense because street signs are designed to be different in three ways: shape, images and/or text, and color.
import cv2
import matplotlib.image as mpimg
def random_blur(x, kernel_size=3):
blur = np.random.randint(1,3)
if blur == 1:
return cv2.GaussianBlur(x, (kernel_size, kernel_size), 0)
if blur == 2:
return x
def random_brighten(x):
dark = np.random.randint(1,3)
if dark == 1:
gamma = np.random.random() * 0.7 + 0.3
if dark == 2:
gamma = np.random.random() * 3.5 + 1.1
invGamma = 1.0 / gamma
table = np.array([((i / 255.0) ** invGamma) * 255 for i in np.arange(0, 256)]).astype("uint8")
return cv2.LUT(x, table)
def random_red_adjust(x):
R = np.random.random() * 0.3 + 0.7
G = np.random.random() * 0.3 + 0.7
B = np.random.random() * 0.3 + 0.7
x[:,:,0] *= R
x[:,:,1] *= G
x[:,:,2] *= B
return x
def random_rotate(x):
degrees = int((np.random.rand() * 80) - 40)
rows = x.shape[0]
cols = x.shape[1]
M = cv2.getRotationMatrix2D((cols/2,rows/2),degrees,1)
dst = cv2.warpAffine(x,M,(cols,rows))
return dst
def random_crop(x):
n = int(x.shape[0] * 0.2)
m = int(x.shape[1] * 0.2)
x1 = np.random.randint(n)
x2 = np.random.randint(n)
y1 = np.random.randint(m)
y2 = np.random.randint(m)
crop = x[x1:x.shape[1]-x2, y1:x.shape[1]-y2]
out = cv2.resize(crop, dsize=(32,32))
return out
def normalize(x):
return x / 255.
def region_of_interest_train(images, sizes, coords):
# the reason for the try statement is to ignore illogical bounding boxes or images lacking them
# the reason for the two if statements is to distinguish between inputs of batch_size and one image
images_roi = []
if len(images.shape) == 4:
for i in range(images.shape[0]):
try:
orig_shape = images[i].shape
resize = cv2.resize(images[i], dsize=tuple(sizes[i]))
crop = resize[coords[i][0]:coords[i][2], coords[i][1]:coords[i][3], :]
rand_crop = random_crop(crop)
rand_bri = random_brighten(rand_crop)
rand_rot = random_rotate(rand_bri)
rand_blu = random_blur(rand_rot)
resize2 = cv2.resize(rand_blu, dsize=tuple(orig_shape[:2]))
norm = normalize(resize2)
out = random_red_adjust(norm)
images_roi.append(out)
except:
images_roi.append(normalize(images[i]))
if len(images.shape) == 3:
orig_shape = images.shape
resize = cv2.resize(images, dsize=tuple(sizes))
crop = resize[coords[0]:coords[2], coords[1]:coords[3], :]
rand_crop = random_crop(crop)
rand_bri = random_brighten(rand_crop)
rand_rot = random_rotate(rand_bri)
rand_blu = random_blur(rand_rot)
resize2 = cv2.resize(rand_blu, dsize=tuple(orig_shape[:2]))
norm = normalize(resize2)
out = random_red_adjust(norm)
images_roi.append(out)
return np.array(images_roi)
def get_batches_train(images, labels, sign_names, sizes, coords, batch_num):
num_images = len(images)
idx = np.arange(num_images)
np.random.shuffle(idx)
for i in range(0, num_images, batch_num):
batch_y = labels[idx[i:i+batch_num]]
batch_sizes = sizes[idx[i:i+batch_num]]
batch_coords = coords[idx[i:i+batch_num]]
batch_names = sign_names[batch_y, 1]
images_x = images[idx[i:i+batch_num]]
batch_x = region_of_interest_train(images_x, batch_sizes, batch_coords).reshape((-1, 32, 32, 3))
yield batch_x, batch_y, batch_names, batch_sizes, batch_coords
# Seperate get_batches and region_of_interest for test and validation without data augmentation
def region_of_interest_test(images, sizes, coords):
images_roi = []
if len(images.shape) == 4:
for i in range(images.shape[0]):
try:
orig_shape = images[i].shape
resize = cv2.resize(images[i], dsize=tuple(sizes[i]))
crop = resize[coords[i][0]:coords[i][2], coords[i][1]:coords[i][3], :]
out = cv2.resize(crop, dsize=tuple(orig_shape[:2]))
images_roi.append(normalize(out))
except:
images_roi.append(normalize(images[i]))
if len(images.shape) == 3:
orig_shape = images.shape
resize = cv2.resize(images, dsize=tuple(sizes))
crop = resize[coords[0]:coords[2], coords[1]:coords[3], :]
out = cv2.resize(crop, dsize=tuple(orig_shape[:2]))
images_roi.append(normalize(out))
return np.array(images_roi)
def get_batches_test(images, labels, sign_names, sizes, coords, batch_num):
num_images = len(images)
for i in range(0, num_images, batch_num):
idx = np.arange(i,i+batch_num)
batch_y = labels[i:i+batch_num]
batch_sizes = sizes[i:i+batch_num]
batch_coords = coords[i:i+batch_num]
batch_names = sign_names[batch_y, 1]
images_x = images[i:i+batch_num]
batch_x = region_of_interest_test(images_x, batch_sizes, batch_coords).reshape((-1, 32, 32, 3))
yield idx, batch_x, batch_y, batch_names, batch_sizes, batch_coords
x = X_train[267]
y = y_train[267]
x_name = sign_names[y]
x_coord = train_coord[267]
x_size = train_sizes[267]
x_resize = cv2.resize(x, dsize=tuple(x_size))
plt.figure(figsize=((15,15)))
plt.suptitle(x_name[1], fontsize=20, y=0.91)
plt.subplot(3,3,1)
plt.title('ORIGINAL')
plt.imshow(x)
for i in range(2,10):
plt.subplot(3,3,i)
plt.imshow(region_of_interest_train(x, x_size, x_coord).reshape((32,32,3)))
Only the train images are randomly augmented, but we'll look at all three sets to make sure everything works properly.
train_roi = region_of_interest_train(X_train, train_sizes, train_coord)
valid_roi = region_of_interest_test(X_valid, valid_sizes, valid_coord)
test_roi = region_of_interest_test(X_test, test_sizes, test_coord)
ii = 3
idx = np.arange(n_train)
np.random.shuffle(idx)
plt.figure(figsize=(15, 6))
plt.suptitle('Train Before', fontsize=16)
for i in range(10):
plt.subplot(2,5,(i+1))
plt.title('{:.33}'.format(sign_names[y_train[idx[i*ii]], 1]))
plt.imshow(X_train[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
plt.figure(figsize=(15, 6))
plt.suptitle('Train After', fontsize=16)
for i in range(10):
plt.subplot(2,5,(i+1))
plt.title('{:.33}'.format(sign_names[y_train[idx[i*ii]], 1]))
plt.imshow(train_roi[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
ii = 3
idx = np.arange(n_validation)
np.random.shuffle(idx)
plt.figure(figsize=(15, 6))
plt.suptitle('Validation Before', fontsize=16)
for i in range(10):
plt.subplot(2,5,(i+1))
plt.title('{:.33}'.format(sign_names[y_valid[idx[i*ii]], 1]))
plt.imshow(X_valid[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
plt.figure(figsize=(15, 6))
plt.suptitle('Validation After', fontsize=16)
for i in range(10):
plt.subplot(2,5,(i+1))
plt.title('{:.33}'.format(sign_names[y_valid[idx[i*ii]], 1]))
plt.imshow(valid_roi[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
ii = 3
idx = np.arange(n_test)
np.random.shuffle(idx)
plt.figure(figsize=(15, 6))
plt.suptitle('Test Before', fontsize=16)
for i in range(10):
plt.subplot(2,5,(i+1))
plt.title('{:.33}'.format(sign_names[y_test[idx[i*ii]], 1]))
plt.imshow(X_test[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
plt.figure(figsize=(15, 6))
plt.suptitle('Test After', fontsize=16)
for i in range(10):
plt.subplot(2,5,(i+1))
plt.title('{:.33}'.format(sign_names[y_test[idx[i*ii]], 1]))
plt.imshow(test_roi[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
ii = 1
idx = np.arange(n_train)
np.random.shuffle(idx)
plt.figure(figsize=(45, 45))
plt.suptitle('80 Train', fontsize=50, y=0.91)
for i in range(10*8):
plt.subplot(2*4,5*2,(i+1))
plt.title('{:.20}'.format(sign_names[y_train[idx[i*ii]], 1]), fontsize=22)
plt.imshow(train_roi[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
ii = 1
idx = np.arange(n_validation)
np.random.shuffle(idx)
plt.figure(figsize=(45, 45))
plt.suptitle('80 Validation', fontsize=50, y=0.91)
for i in range(10*8):
plt.subplot(2*4,5*2,(i+1))
plt.title('{:.33}'.format(sign_names[y_valid[idx[i*ii]], 1]), fontsize=22)
plt.imshow(valid_roi[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
ii = 1
idx = np.arange(n_test)
np.random.shuffle(idx)
plt.figure(figsize=(45, 45))
plt.suptitle('80 Test', fontsize=50, y=0.91)
for i in range(10*8):
plt.subplot(2*4,5*2,(i+1))
plt.title('{:.33}'.format(sign_names[y_test[idx[i*ii]], 1]), fontsize=22)
plt.imshow(test_roi[idx[i*ii]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
import tensorflow as tf
from collections import namedtuple
def build_network(n_classes, batch_num, size_mult=64, alpha=0.2):
tf.reset_default_graph()
drop_rate = tf.placeholder(tf.float32, name='drop_rate')
is_train = tf.placeholder(tf.bool, name='is_train')
learn_rate = tf.placeholder(tf.float32, name='learn_rate')
with tf.name_scope('inputs'):
x = tf.placeholder(tf.float32, [None, 32, 32, 3], name='inputs')
with tf.name_scope('targets'):
targets = tf.placeholder(tf.int32, [None])
y_one_hot = tf.one_hot(targets, n_classes, name='y_one_hot')
y_reshaped = tf.reshape(y_one_hot, [-1, n_classes])
with tf.name_scope('CONV1'):
x1 = tf.layers.conv2d(x, 2*size_mult, 5, strides=1, padding='same', use_bias=True, name='conv1')
relu1 = tf.maximum(alpha * x1, x1)
with tf.name_scope('CONV2'):
x2 = tf.layers.conv2d(relu1, 2*size_mult, 5, strides=2, padding='same', use_bias=True, name='conv2')
relu2 = tf.maximum(alpha * x2, x2)
with tf.name_scope('CONV3'):
x3 = tf.layers.conv2d(relu2, 4*size_mult, 5, strides=1, padding='same', use_bias=False, name='conv3')
bn3 = tf.layers.batch_normalization(x3, training=is_train, name='batch3')
relu3 = tf.maximum(alpha * bn3, bn3)
do3 = tf.layers.dropout(relu3, rate=drop_rate)
with tf.name_scope('CONV4'):
x4 = tf.layers.conv2d(do3, 4*size_mult, 5, strides=2, padding='same', use_bias=True, name='conv4')
relu4 = tf.maximum(alpha * x4, x4)
with tf.name_scope('CONV5'):
x5 = tf.layers.conv2d(relu4, 8*size_mult, 3, strides=1, padding='same', use_bias=True, name='conv5')
relu5 = tf.maximum(alpha * x5, x5)
do5 = tf.layers.dropout(relu5, rate=drop_rate)
with tf.name_scope('CONV6'):
x6 = tf.layers.conv2d(do5, 8*size_mult, 3, strides=2, padding='same', use_bias=True, name='conv6')
relu6 = tf.maximum(alpha * x6, x6)
with tf.name_scope('CONV7'):
x7 = tf.layers.conv2d(relu6, 8*size_mult, 3, strides=1, padding='same', use_bias=False, name='conv7')
bn7 = tf.layers.batch_normalization(x7, training=is_train, name='batch7')
relu7 = tf.maximum(alpha * bn7, bn7)
do7 = tf.layers.dropout(relu7, rate=drop_rate)
with tf.name_scope('FC8'):
x8 = tf.contrib.layers.flatten(do7)
fc8 = tf.layers.dense(x8, 16 * size_mult, use_bias=True, activation=None, name='fc8')
relu8 = tf.maximum(alpha * fc8, fc8)
do8 = tf.layers.dropout(relu8, rate=drop_rate)
with tf.name_scope('logits'):
logits = tf.layers.dense(do8, n_classes, use_bias=True, activation=None, name='logits')
with tf.name_scope('predictions'):
preds = tf.nn.softmax(logits, name='predictions')
correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(y_reshaped, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')
with tf.name_scope('cost'):
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_reshaped), name='cost')
with tf.name_scope('train'):
train_opt = tf.train.AdamOptimizer(learn_rate).minimize(cost)
export_nodes = ['x', 'targets', 'drop_rate', 'is_train', 'learn_rate', 'relu1', 'relu2', 'relu3',
'relu4', 'relu5', 'relu6', 'relu7', 'correct_pred', 'preds', 'cost', 'accuracy', 'train_opt']
Graph = namedtuple('Graph', export_nodes)
local_dict = locals()
graph = Graph(*[local_dict[each] for each in export_nodes])
return graph
how_many = 5
def top_n_predictions(pred, top_n=how_many, probs_off=True):
values, indices = tf.nn.top_k(pred, k=top_n)
val = np.squeeze(values.eval()).tolist()
ind = np.squeeze(indices.eval()).tolist()
if probs_off:
out = sign_names[ind, 1]
else:
out = val
return out
def view_val_with_preds(sess, image, label, sign_name, size, coord, top_n=how_many):
sample_x = region_of_interest_test(image, size, coord)
feed = {model.x: sample_x.reshape((1,32,32,-1)),
model.targets: label.reshape((1,)),
model.drop_rate: 0.,
model.is_train: False}
pred = sess.run(model.preds, feed_dict=feed)
top_n_pred_names = top_n_predictions(pred)
top_n_pred_probs = top_n_predictions(pred, probs_off=False)
plt.figure(figsize=(10, 8))
plt.suptitle('{:.33}'.format(sign_name), fontsize=16)
plt.subplot(2,2,1)
y_pos = np.arange(top_n)
probabilities = np.flip(top_n_pred_probs, 0)
plt.barh(y_pos, probabilities, align='center', color='blue', tick_label=np.flip(top_n_pred_names, 0))
plt.tick_params(axis='x', which='both', bottom='off', top='off', labelbottom='off')
plt.xlabel('Probability')
plt.subplot(2,2,2)
plt.imshow(image)
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
plt.subplots_adjust(wspace=0.75)
plt.show()
epochs = 103
batch_size = 32
dropout_rate = 0.4
learn_rate = 0.0001
model = build_network(n_classes, learn_rate, batch_size)
saver = tf.train.Saver()
n_batches = int(n_train/batch_size)
steps = 0
train_losses, batch_accs = [], []
total_wrong_indexes = []
print_every = 200
new_val_acc = 0
I find it useful to keep an eye on the network as it trains and to offer words of encouragement. Intuitively, this doesn't make sense. But, like dropout, it works well in practice.
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for e in range(epochs):
for x, y, _, _, _ in get_batches_train(X_train, y_train, sign_names, train_sizes, train_coord, batch_size):
steps += 1
feed = {model.x: x,
model.targets: y,
model.drop_rate: dropout_rate,
model.is_train: True,
model.learn_rate: learn_rate}
batch_acc, batch_loss, _ = sess.run([model.accuracy, model.cost, model.train_opt], feed_dict=feed)
train_losses.append(batch_loss)
batch_accs.append(batch_acc)
if steps % print_every == 0:
avg_loss = np.mean(train_losses)
avg_acc = np.mean(batch_accs)
print('Epoch {}/{}. . . Train Loss {:.4}. . . Train Acc {:.4}'.format(e+1, epochs, avg_loss, avg_acc))
print('')
print('Caclulating loss & accuracy on validation set...')
val_losses, val_accs, wrong_indexes = [], [], []
for val_idx, val_x, val_y, val_names, _, _ in get_batches_test(X_valid, y_valid, sign_names,
valid_sizes, valid_coord, batch_size):
feed = {model.x: val_x,
model.targets: val_y,
model.drop_rate: 0.,
model.is_train: False}
val_acc, val_cor, val_loss = sess.run([model.accuracy, model.correct_pred, model.cost], feed_dict=feed)
wrong_idx = [val_idx[i] for i in range(len(val_cor)) if val_cor[i] == False]
wrong_indexes.append(wrong_idx)
val_losses.append(val_loss)
val_accs.append(val_acc)
avg_val_loss = np.mean(val_losses)
avg_val_acc = np.mean(val_accs)
print('Epoch {}/{}. . . Val Loss {:.4}. . . Val Acc {:.4}'.format(e+1, epochs, avg_val_loss, avg_val_acc))
# Save only checkpoints with a validation accuracy over 90%
if avg_val_acc > 0.90:
if new_val_acc == 0:
!mkdir checks
if avg_val_acc > new_val_acc:
new_val_acc = avg_val_acc
saver.save(sess, './checks/tsigns.ckpt')
print('Model Saved!')
print('')
wrong_indexes = np.array([p for sublist in wrong_indexes for p in sublist])
ind, cnt = np.unique(y_valid[wrong_indexes], return_counts=True)
wrong_names = [sign_names[ind[i], 1] for i in range(len(ind))]
for i in range(len(cnt)):
# Only print sign names of those wrong 5% or more of time
if (cnt[i]/np.sum(cnt))*100 > 4.9:
print('<{:.20}> is {:.0f}% of total wrong'.format(wrong_names[i], (cnt[i]/np.sum(cnt))*100))
print('')
total_wrong_indexes.append(wrong_indexes)
# Every 10 epochs, let's look at some validation images and the top-5 predicted labels
if (e+1) % 10 == 0:
print('Displaying validation samples...')
num_images = len(X_valid)
idx = np.arange(num_images)
np.random.shuffle(idx)
num_show = 5
[view_val_with_preds(sess, X_valid[idx[i]], y_valid[idx[i]], sign_names[y_valid[idx[i]], 1],
valid_sizes[idx[i]], valid_coord[idx[i]]) for i in range(num_show)]
#delete above and use below to only view images that were misclassified:
#num_images = len(wrong_indexes)
#idx = np.arange(num_images)
#np.random.shuffle(idx)
#num_show = 10
#[view_val_with_preds(sess, X_valid[wrong_indexes[idx[i]]], y_valid[wrong_indexes[idx[i]]],
# sign_names[y_valid[wrong_indexes[idx[i]]], 1], valid_sizes[wrong_indexes[idx[i]]],
# valid_coord[wrong_indexes[idx[i]]]) for i in range(num_show)]
print('Model finished training :)')
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
test_accs, wrong_indices = [], []
for test_idx, test_x, test_y, test_names, _, _ in get_batches_test(X_valid, y_valid, sign_names,
valid_sizes, valid_coord, batch_size):
feed = {model.x: test_x,
model.targets: test_y,
model.drop_rate: 0.,
model.is_train: False}
test_acc, test_cor = sess.run([model.accuracy, model.correct_pred], feed_dict=feed)
test_accs.append(test_acc)
wrong_idx = [test_idx[i] for i in range(len(test_cor)) if test_cor[i] == False]
wrong_indices.append(wrong_idx)
avg_test_acc = np.mean(test_accs)
print('Validation Accuracy {:.4}'.format(avg_test_acc))
print('Displaying validation samples...')
num_images = len(X_valid)
idx = np.arange(num_images)
np.random.shuffle(idx)
num_show = 5
[view_val_with_preds(sess, X_valid[idx[i]], y_valid[idx[i]], sign_names[y_valid[idx[i]], 1],
valid_sizes[idx[i]], valid_coord[idx[i]]) for i in range(num_show)]
wrong_indices = np.array([p for sublist in wrong_indices for p in sublist])
n_wrong = len(wrong_indices)
print('Total misclassified signs in validation set: {}'.format(n_wrong))
plt.figure(figsize=(45, 45))
for i in range(80):
plt.subplot(8,10,(i+1))
plt.title('{:.20}'.format(sign_names[y_valid[wrong_indices[i]], 1]), fontsize=22)
plt.imshow(X_valid[wrong_indices[i]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
I'll also track the true positives, true negatives, false positives, and false negatives, then calculate the precision, recall, and specificity for each class. This can help determine how to augment data by showing the network's weak labels.
tp = np.zeros(n_classes)
tn = np.zeros(n_classes)
fp = np.zeros(n_classes)
fn = np.zeros(n_classes)
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
test_accs, wrong_indices = [], []
for test_idx, test_x, test_y, test_names, _, _ in get_batches_test(X_test, y_test, sign_names,
test_sizes, test_coord, batch_size):
feed = {model.x: test_x,
model.targets: test_y,
model.drop_rate: 0.,
model.is_train: False}
test_acc, test_preds, test_cor = sess.run([model.accuracy, model.preds, model.correct_pred], feed_dict=feed)
test_preds = tf.argmax(test_preds, 1).eval()
for i in range(len(test_cor)):
if test_cor[i] == True:
tp[test_y[i]] += 1
for j in range(n_classes):
if j != test_y[i]:
tn[j] += 1
if test_cor[i] == False:
fp[test_preds[i]] += 1
fn[y_test[i]] += 1
test_accs.append(test_acc)
wrong_idx = [test_idx[i] for i in range(len(test_cor)) if test_cor[i] == False]
wrong_indices.append(wrong_idx)
avg_test_acc = np.mean(test_accs)
print('Test Accuracy {:.4}'.format(avg_test_acc))
print('Displaying test samples...')
num_images = len(X_test)
idx = np.arange(num_images)
np.random.shuffle(idx)
num_show = 5
[view_val_with_preds(sess, X_test[idx[i]], y_test[idx[i]], sign_names[y_test[idx[i]], 1],
test_sizes[idx[i]], test_coord[idx[i]]) for i in range(num_show)]
wrong_indices = np.array([p for sublist in wrong_indices for p in sublist])
n_wrong = len(wrong_indices)
print('Total misclassified signs in test set: {}'.format(n_wrong))
print('')
precision = tp / (tp + fp)
recall = tp / (tp + fn)
specificity = tn / (tn + fp)
accuracy = np.sum(tp) / len(X_test)
print('Double-check accuracy: {:.4}'.format(accuracy))
print('')
for i in range(n_classes):
print('{}:'.format(sign_names[i, 1]))
print(' precision: {:.4}'.format(precision[i]))
print(' recall: {:.4}'.format(recall[i]))
print(' specificity: {:.4}'.format(specificity[i]))
print('')
plt.figure(figsize=(45, 45))
for i in range(100):
plt.subplot(10,10,(i+1))
plt.title('{:.20}'.format(sign_names[y_test[wrong_indices[i+100]], 1]), fontsize=22)
plt.imshow(X_test[wrong_indices[i+100]])
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
eg1 = normalize(cv2.resize(mpimg.imread('traffic-signs-web/example_1.jpg'), dsize=(32,32)))
eg2 = normalize(cv2.resize(mpimg.imread('traffic-signs-web/example_2.jpg'), dsize=(32,32)))
eg3 = normalize(cv2.resize(mpimg.imread('traffic-signs-web/example_4.jpg'), dsize=(32,32)))
eg4 = normalize(cv2.resize(mpimg.imread('traffic-signs-web/example_5.jpg'), dsize=(32,32)))
eg5 = normalize(cv2.resize(mpimg.imread('traffic-signs-web/example_6.jpg'), dsize=(32,32)))
egs = [eg1, eg2, eg3, eg4, eg5]
plt.figure(figsize=(15,5))
for i in range(5):
plt.subplot(1,5,(i+1))
plt.imshow(egs[i])
def view_web_with_preds(sess, image, top_n=how_many):
feed = {model.x: image.reshape((1,32,32,-1)),
model.drop_rate: 0.,
model.is_train: False}
pred = sess.run(model.preds, feed_dict=feed)
top_n_pred_names = top_n_predictions(pred)
top_n_pred_probs = top_n_predictions(pred, probs_off=False)
plt.figure(figsize=(10, 8))
plt.subplot(2,2,1)
y_pos = np.arange(top_n)
probabilities = np.flip(top_n_pred_probs, 0)
plt.barh(y_pos, probabilities, align='center', color='blue', tick_label=np.flip(top_n_pred_names, 0))
plt.xlabel('Probability')
plt.subplot(2,2,2)
plt.imshow(image)
plt.tick_params(axis='both', which='both', bottom='off', top='off', labelbottom='off')
plt.subplots_adjust(wspace=0.75)
plt.show()
return np.flip(probabilities, 0), top_n_pred_names
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
for i in range(5):
probs, top_n_names = view_web_with_preds(sess, egs[i])
print('Top 5 Softmax Probabilities:')
[print(' {}: {:.4}'.format(top_n_names[i], probs[i])) for i in range(5)]
print('{}:'.format(sign_names[20, 1]))
print(' precision: {:.4}'.format(precision[20]))
print(' recall: {:.4}'.format(recall[20]))
print(' specificity: {:.4}'.format(specificity[20]))
dang_curve_to_right = normalize(cv2.resize(mpimg.imread('traffic-signs-web/dangerous_curve_to_right.jpg'), dsize=(32,32)))
dang_curve_to_right2 = normalize(cv2.resize(mpimg.imread('traffic-signs-web/dangerous_curve_to_right2.jpg'), dsize=(32,32)))
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
probs, top_n_names = view_web_with_preds(sess, dang_curve_to_right)
print('Top 5 Softmax Probabilities:')
[print(' {}: {:.4}'.format(top_n_names[i], probs[i])) for i in range(5)]
probs, top_n_names = view_web_with_preds(sess, dang_curve_to_right2)
print('Top 5 Softmax Probabilities:')
[print(' {}: {:.4}'.format(top_n_names[i], probs[i])) for i in range(5)]
Well, at least the miss was predictable.
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
feed = {model.x: egs[0].reshape((1,32,32,-1)),
model.drop_rate: 0.,
model.is_train: False}
relu1, relu2 = sess.run([model.relu1, model.relu2], feed_dict=feed)
plt.figure(figsize=(5,5))
plt.imshow(egs[0])
plt.title('Original Image')
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 1', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu1[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 2', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu2[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
feed = {model.x: egs[-1].reshape((1,32,32,-1)),
model.drop_rate: 0.,
model.is_train: False}
relu1, relu2 = sess.run([model.relu1, model.relu2], feed_dict=feed)
plt.figure(figsize=(5,5))
plt.imshow(egs[-1])
plt.title('Original Image')
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 1', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu1[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 2', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu2[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))
Keeping in mind how this sign had low precision and recall, notice how the network has trouble recognizing the relevant line that distinguishes it from other triangular street signs.
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
feed = {model.x: dang_curve_to_right2.reshape((1,32,32,-1)),
model.drop_rate: 0.,
model.is_train: False}
relu1, relu2 = sess.run([model.relu1, model.relu2], feed_dict=feed)
plt.figure(figsize=(5,5))
plt.imshow(dang_curve_to_right2)
plt.title('Original Image')
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 1', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu1[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 2', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu2[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))
Because why not?
van_gogh = normalize(cv2.resize(mpimg.imread('traffic-signs-web/van_gogh.jpg'), dsize=(32,32)))
van_gogh_orig = mpimg.imread('traffic-signs-web/van_gogh.jpg')
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('checks'))
feed = {model.x: van_gogh.reshape((1,32,32,-1)),
model.drop_rate: 0.,
model.is_train: False}
relu1, relu2 = sess.run([model.relu1, model.relu2], feed_dict=feed)
plt.figure(figsize=(5,5))
plt.imshow(van_gogh_orig)
plt.title('Original Image')
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 1', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu1[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))
plt.figure(figsize=(35,35))
plt.suptitle('ReLU 2', fontsize=45, y=0.91)
for i in range(64):
plt.subplot(8,8,(i+1))
plt.imshow(relu2[0, :, :, i], interpolation='nearest', cmap='gray')
plt.title('FeatureMap #{}'.format(i))